#!/usr/bin/python
#coding=utf-8
import sys
import os
reload(sys)
sys.setdefaultencoding("utf8")
import gevent
from gevent import monkey
monkey.patch_all()
from gevent.queue import Queue, Empty
from gevent.pool import Pool
import requests
import re
from pyquery import PyQuery as pq

localDir = '/Users/mac/Pictures/zgp/'
tasks = Queue()

def putUrl(url):
    i = 0
    while (True):
        try:
            headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                       'Accept-Encoding':'gzip, deflate, sdch',
                        'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
                        'Connection':'keep-alive',
                        'Host':'www.zhaogepu.com','Referer':'http://www.zhaogepu.com/jianpu/1.html',
                        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36'
                       }
            res = requests.get(url,headers=headers,timeout=20)
            res.encoding = 'utf-8'
            if res.status_code == 200 :
                doc = pq(res.text)
                zurl = doc(".images_list").find('a').attr('href')
                title = doc(".images_list").find('a').attr('title')
                index = re.findall(r'\d+',url)[0] #索引
                try:
                    ext = zurl[zurl.rfind('.'):] #文件扩展
                except Exception,e:
                    print '是文本忽略下载'
                filePath = '%s%s.%s%s' % (localDir,index,title,ext)
                if os.path.isfile(filePath):
                    print '%s:%s文件已存在' %(index,title)
                    break
                if zurl :
                    tasks.put_nowait([zurl,index,title,ext])
            else:
                print 'url:%s访问失败,失败原因:%d' %(url,res.status_code)
                break
        except Exception,e:
            i = i+1
            print 'url:%s访问超时,正在重试第%s' %(url,i)

def getUrl():
    while True:
        url = tasks.get()
        print url

def boss():
    """
    Boss will wait to hand out work until a individual worker is
    free since the maxsize of the task queue is 3.
    """
    listUrls = []
    gpool = Pool(2)
    for i in range(1,1000):
        listUrls.append('http://www.zhaogepu.com/jianpu/%s.html' %(i))
    jobs = [gpool.spawn(putUrl,url) for url in listUrls]
    gevent.joinall(jobs)

gevent.joinall([
    gevent.spawn(boss),
    gevent.spawn(getUrl),
])